"use strict"

/**
 * 调度模块，主要获取所有的爬虫定义，然后分别调用爬虫，最后将数据保存，并发送邮件
 *  
 */

var fs = require("fs");
var q = require("q");
require('date-utils');

var spider = require("../spider");
var db = require("../db");
var mail = require("../mail");
var indexer = require("../index");


const MAIL_NOTIFY_BEFORE_DAYS = 9;
const MAIL_NOTIFY_ITEM_COUNT = 1000;

console.log(spider.name + " version: " + spider.version);
console.log("---------------------------------------");

var schdlObject = {
    version: "1.0.0",
    name: "crawler scheduler",
    db: db,
    mail: mail,
    option: "all",
    testmodule: null //测试模块
};

var modules_files = fs.readdirSync(__dirname + "/../modules");
var modules = [];

// 载入所有有效的爬虫模块
var load_modules = function () {
    for (var i = 0, j = modules_files.length; i < j; i++) {
        var fn = modules_files[i];
        if (/.js$/.test(fn)) {
            var module_name = fn.substring(0, fn.length - 3);
            if (module_name == "base") continue;

            var m = require("../modules/" + module_name);
            m.module_name = module_name;
            if (m) {
                console.log((m["name"] || "") + " [" + module_name + "] loaded.");
                if (m.hasOwnProperty('enabled')) {
                    if (m.enabled == true)
                        modules.push(m);
                } else {
                    modules.push(m);
                }
            }
        }
    }
    console.log("spirder count [avaliable]: " + modules.length);
};

// 打开数据库
var db_open = function () {
    db.open();
    return 1;
};

// 关闭数据库
var db_close = function () {
    db.close();
};

// 判断是否需要保存
var recordHanlder = function (item) {
    return function () {
        var title = item.title;
        var d = q.defer();
        db.find({ "title": title }, function (err, recs) {
            if (err) {
                console.error(err);
                d.resolve();
            } else {
                if (recs.length == 0) {
                    db.insert(item, function (err, result) {
                        // result like {"fieldCount":0,"affectedRows":1,"insertId":6317,"serverStatus":2,"warningCount":0,"message":"","protocol41":true,"changedRows":0}
                        if (err)
                            console.error(err);
                        else
                            console.log(title + " has inserted");
                        if (!result) {
                            console.error("insert into falt, result return undefined");
                            d.resolve();
                            return;
                        }
                        var id = result.insertId; // bug fixed 如果没有更新id，插入到solr因为类型(int)不对而报异常
                        var getOneItemSql = "select * from items where id=" + id + " limit 1";
                        db.query(getOneItemSql, function (err, items) {
                            if (err) {
                                console.error(err);
                                d.resolve();
                            } else {
                                var oneItem = items[0];
                                delete oneItem.ext;
                                delete oneItem.processed;
                                // 添加索引
                                indexer.add(oneItem, function (err, obj) {
                                    d.resolve();
                                });
                            }
                        });
                    });
                } else {
                    console.log(title + " has exists");
                    d.resolve();
                }
            }
        });
        return d.promise;
    };
};

// 每个条目的处理
var itemHandle = function (items) {
    var result = q('');
    for (var i = 0, j = items.length; i < j; i++) {
        var item = items[i];
        var title = item.title;
        console.log((i + 1) + " " + title);
        if (schdlObject.testmodule) {
            //do nothing;
        } else {
            result = result.then(recordHanlder(item));
        }
    }
    return result;
};

// 每个爬虫的处理
var moduleHandle = function (m) {
    return function () {
        console.log(m.name + " " + m.version);
        var d = q.defer();
        m.getUrls(spider).then(itemHandle).then(function () {
            d.resolve();
            console.log("=================");
        });
        return d.promise;
    };
};

// 发送邮件逻辑
var mailHander = function (rec) {
    return function () {
        var d = q.defer();
        var tip = rec.tip;
        var kys = rec.keywords;
        kys = kys.replace(/,/g, " ").replace(/，/g, " "); // 容错，历史设置
        var toaddr = rec.emails;
        var id = rec.id;
        var content = "";
        var queryTerm = { "title": kys };
        var now = Date.tomorrow();
        var startDay = indexer.getBeforeDate(MAIL_NOTIFY_BEFORE_DAYS);
        queryTerm.create_time = {
            start: startDay,
            end: now,
            field: "create_time"
        };
        var sql = "";
        indexer.query(queryTerm, 0, MAIL_NOTIFY_ITEM_COUNT, function (err, data) {
            if (err || data == null || data.response.docs.length == 0) {
                sql = "select * from items where 1=0";
            } else {
                var items = data.response.docs;
                var ids = [];
                for (var i = 0, j = items.length; i < j; i++) {
                    var item = items[i];
                    ids.push(item.id);
                }
                sql = "select * from items where id in (" + ids.join(",") + ")";
            }

            sql = "select a.*, b.mailid, b.itemid from (" + sql + ") a left join (select * from mail2items where mailid=" + id + ") b on (a.id=b.itemid)";
            sql = "select * from (" + sql + ") c where c.itemid is null";
            var insertSql = null;
            db.query(sql, function (err, recs) {
                if (!recs || recs.length == 0) {
                    console.log("没有发现相关的商机\r\nall finished.");
                } else {
                    insertSql = "insert into mail2items (mailid, itemid) values "
                    recs.forEach(function (_rec) {
                        content = content + _rec.title + " " + _rec.url + "\r\n";
                        insertSql += "(" + id + ", " + _rec.id + "),";
                    });
                    insertSql = insertSql.substring(0, insertSql.length - 1);
                }

                if (content != "") {
                    mail.text = "以下内容是[" + tip + "]，详细内容请注意邮件中的地址。\r\n名称\t\地址\r\n\r\n" + content + "\r\n以上内容从网络自动获取(每日自动更新)，供内部参考！";
                    mail.sendMail(toaddr, function (error, msg) {
                        if (error) {
                            console.error(error);
                            d.resolve();
                        } else {
                            if (insertSql) {
                                db.query(insertSql, function (err, result) {
                                    console.log("邮件发送成功\r\nall finished.");
                                    d.resolve();
                                });
                            } else {
                                d.resolve();
                            }
                        }
                    });
                } else {
                    d.resolve();
                }

            });
        });
        return d.promise;
    };
};

var sendMail = function () {
    console.log("---- send mail ----");
    var d = q.defer();
    var content = "";
    var Q = q('');
    db.query("select * from mails", function (err, recs) {
        recs.forEach(function (rec) {
            Q = Q.then(mailHander(rec));
        });
        Q.then(function () {
            console.log("邮件发送成功\r\nall finished.");
            d.resolve();
        });
    });
    return d.promise;
};

// main
var start = function () {
    if (schdlObject.option == "exit") {
        return;
    }


    var db_result = db_open();

    load_modules();

    var test_module = schdlObject.testmodule;
    var result = q('');
    if (schdlObject.option == "newitem" || schdlObject.option == "all") {
        for (var i = 0, j = modules.length; i < j; i++) {
            var m = modules[i];
            if (test_module) {
                if (test_module == m.module_name) {
                    result = result.then(moduleHandle(m));
                }
            } else {
                result = result.then(moduleHandle(m));
            }
        }
    }

    if (schdlObject.option == "send" || schdlObject.option == "all") {
        result = result.then(sendMail);
    }

    if (schdlObject.option == "index") {
        result = result.then(function () {
            var def = q.defer();
            indexer.indexAll(function () {
                def.resolve();
            });
            return def.promise;
        });
    }

    result.then(function () { // end(exit) point
        if (db_result)
            db_close();
    });
};

schdlObject.start = start;

exports = module.exports = schdlObject;